# 多进程爬虫
import requests
from multiprocessing import Process, Lock
# 队列模块必须使用multiprocessing中的
from multiprocessing import Queue
import time
from fake_useragent import UserAgent

class DoubanSpider:
    def __init__(self):
        self.url = 'https://movie.douban.com/j/chart/top_list?type=11&interval_id=100%3A90&action=&start={}&limit=20'
        # 队列、线程锁
        self.q = Queue()
        self.lock = Lock()

    def get_headers(self):
        """功能函数"""
        headers = {'User-Agent':UserAgent(path='fake_useragent.json').random}

        return headers

    def get_html(self, url):
        """功能函数"""
        headers = self.get_headers()
        html = requests.get(url=url, headers=headers).json()

        return html

    def url_to_q(self):
        """生成所有的url地址入队列"""
        for page in range(0, 762, 20):
            page_url = self.url.format(page)
            # 入队列
            self.q.put(page_url)

    def parse_html(self):
        """进程事件函数"""
        while True:
            # 如果队列不为空
            # self.q: ['http://page8.html']
            # Thread-1: 队列不为空，条件成立
            # Thread-2: 队列不为空，条件也成立
            # 操作全局变量 - 加线程锁
            self.lock.acquire()
            if not self.q.empty():
                url = self.q.get()
                # 释放锁
                self.lock.release()
                html = self.get_html(url)
                for one_dict in html:
                    item = {}
                    item['title'] = one_dict['title']
                    item['score'] = one_dict['score']
                    print(item)
            else:
                # 释放锁
                self.lock.release()
                break

    def crawl(self):
        """逻辑函数"""
        # 1.先让url地址入队列
        self.url_to_q()
        # 2.创建多线程
        t_list = []
        for i in range(4):
            t = Process(target=self.parse_html)
            t_list.append(t)
            t.start()

        for t in t_list:
            t.join()

if __name__ == '__main__':
    start = time.time()
    spider = DoubanSpider()
    spider.crawl()
    end = time.time()
    print('time: %.2f' % (end - start))



















